import requests
from lxml import etree
headers ={
    "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"
}
url1 ="https://www.bbiquge.net/book_682/"

respones = requests.get(url1,headers=headers)
html = etree.HTML(respones.content.decode("gbk"))

dd_list = html.xpath('//div[@class="zjbox"]/dl[@class="zjlist"]/dd')
for dd in dd_list:
    href =url1+dd.xpath('./a/@href')[0]
    title = dd.xpath('./a/text()')[0]
    resp = requests.get(href,headers=headers,timeout=5)
    html = etree.HTML(resp.content.decode("gbk"))
    content = html.xpath('//div[@id="content"]/text()')
    c=''
    for i in content:
        c=c+i
    print(title)
    with open("./data/斗破苍穹/{}.text".format(title),"w",encoding="utf-8")as f:
         f.write(c)